#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
import os
from spider.requests_helper import Request
from lxml import html


def get_total_page():
    url = "http://www.meizitu.org/"
    # 获得html内容
    html_text = Request().get_html_text(url)

    # 获得xpath对象
    selector = html.fromstring(html_text)
    total_page = selector.xpath('//div[@class="text-center"]/ul/li[last()-1]/a/text()')
    return int(total_page[0])


def spider(page):
    """ 爬取meizitu网站的图片信息 """
    url = "http://www.meizitu.org/page/{page}/".format(page=page)

    # 获得html内容
    html_text = Request().get_html_text(url)

    # 获得xpath对象
    selector = html.fromstring(html_text)

    thumb_list = selector.xpath('//div[@class="thumb"]')
    img_list = []
    for thumb in thumb_list:
        img = {}
        # 获得图片地址
        src = thumb.xpath('img/@src')
        # 获得图片名称
        name = thumb.xpath('div/div/a/text()')
        img['src'] = src[0]
        img['name'] = name[1].strip()
        img_list.append(img)

    return img_list


def download_img(url, path):
    img = requests.get(url)
    index = path.rindex("/")
    folder = path[:index]
    if not os.path.exists(folder):
        os.mkdir(folder)

    with open(path, 'wb') as f:
        f.write(img.content)


if __name__ == '__main__':
    path = 'G:/meizitu/page{page}/{name}.jpg'
    print("开始爬取www.meizitu.org网站图片")
    for page in range(get_total_page()):
        page = page + 1
        img_list = spider(page)
        print("开始爬取第%s页数据" % page)
        for img in img_list:
            file = path.format(page=page, name=img['name'])
            download_img(img['src'], file)
            print("爬取成功地址: ", file)
        print("爬取第%s页数据结束" % page)

    print("爬取www.meizitu.org网站图片结束")


